/*
 *             Automatically Tuned Linear Algebra Software v3.8.3
 *                      (C) Copyright 2001 Julian Ruhe
 *
 * Code contributers : Julian Ruhe, Peter Soendergaard
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions, and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *   3. The name of the ATLAS group or the names of its contributers may
 *      not be used to endorse or promote products derived from this
 *      software without specific written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ATLAS GROUP OR ITS CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 */

/****************************************************************************/
/* This routine originally written in NASM by Julian Ruhe.  See             */
/*    ATLAS/tune/blas/gemm/CASES/objs/ATL_dJIK30x30x30TN30x30x0_a1_b1.asm   */
/* for the original.  Translated into gnu inline assembly (to simplify the  */
/* ATLAS build process) and extended to all precisions by                   */
/* Peter Soendergaard.  Edited by Clint Whaley so it runs on any x87 proc   */
/* (got rid of prefetch and FEMMS for certain archs)                        */
/****************************************************************************/
#include "atlas_misc.h"
#ifndef ATL_GAS_x8632
   #error "This kernel requires gas x86-32 assembler!"
#endif
#if defined(ATL_3DNow)
   #define PREFETCH  prefetch
   #define PREFETCHW prefetchw
   #define USE_PREFETCH
   #define NOFEMMS
#elif defined(ATL_SSE1) || defined (ATL_SSE2)
   #define PREFETCH prefetchnta
   #define PREFETCHW prefetchnta
   #define USE_PREFETCH
   #define NOFEMMS
#else
   #define NOFEMMS
#endif

#if !defined(MB) || !defined(KB) || MB != 30 || KB != 30
   #error "MB and KB must be 30!!"
#endif

#define ASM __asm__ __volatile__


/* Since the code only works for NB==30 this is hardcoded. */
#define PNB "30"

#if defined(DREAL) || defined(DCPLX)
#define DS "8"

/* These macros define memory-register operations for doubles. */
#define ADD "faddl"
#define MUL "fmull"
#define LOAD "fldl"
#define STO "fstpl"

/* These macros indicates where the parameters are placed on the stack. */
#define Nstack "0xc(%%ebp)"
#define Astack "0x1c(%%ebp)"
#define Bstack "0x24(%%ebp)"
#define BETAstack "0x2c(%%ebp)"
#define Cstack "0x34(%%ebp)"
#define LDCstack "0x38(%%ebp)"

#ifdef BETAX
/* This variable is placed here to make it go into the data segment
   instead of the stack */
static double locbeta;
#endif

/* Define DSC as the length of a complex number (stride in C). */
#ifdef DCPLX
#define DSC "16"
#else
#define DSC DS
#endif


#endif

#if defined(SREAL) || defined(SCPLX)
#define DS "4"

/* These macros define memory-register operations for floats. */
#define ADD "fadds"
#define MUL "fmuls"
#define LOAD "flds"
#define STO "fstps"

/* These macros indicates where the parameters are placed on the stack. */
#define Nstack "12(%%ebp)"
#define Astack "24(%%ebp)"
#define Bstack "32(%%ebp)"
#define BETAstack "40(%%ebp)"
#define Cstack "44(%%ebp)"
#define LDCstack "48(%%ebp)"

#ifdef BETAX
static float locbeta;
#endif

/* Define DSC as the length of a complex number (stride in C). */
#ifdef SCPLX
#define DSC "8"
#else
#define DSC DS
#endif


#endif

#ifdef DCPLX

#define CPTR0 " -15*" DS
#define CPTR1 " -13*" DS
#define CPTR2 " -11*" DS
#define CPTR3 " -9*" DS
#define CPTR4 " -7*" DS
#define CPTR5 " -5*" DS

#define CPTR6 " -3*" DS
#define CPTR7 " -1*" DS
#define CPTR8 " 1*" DS
#define CPTR9 " 3*" DS
#define CPTR10 " 5*" DS
#define CPTR11 " 7*" DS

#define CPTR12 " -15*" DS
#define CPTR13 " -13*" DS
#define CPTR14 " -11*" DS
#define CPTR15 " -9*" DS
#define CPTR16 " -7*" DS
#define CPTR17 " -5*" DS

#define CPTR18 " -3*" DS
#define CPTR19 " -1*" DS
#define CPTR20 " 1*" DS
#define CPTR21 " 3*" DS
#define CPTR22 " 5*" DS
#define CPTR23 " 7*" DS

#define CPTR24 " -15*" DS
#define CPTR25 " -13*" DS
#define CPTR26 " -11*" DS
#define CPTR27 " -9*" DS
#define CPTR28 " -7*" DS
#define CPTR29 " -5*" DS

#else

#define CPTR0 " -15*" DSC
#define CPTR1 " -14*" DSC
#define CPTR2 " -13*" DSC
#define CPTR3 " -12*" DSC
#define CPTR4 " -11*" DSC
#define CPTR5 " -10*" DSC
#define CPTR6 " -9*" DSC
#define CPTR7 " -8*" DSC
#define CPTR8 " -7*" DSC
#define CPTR9 " -6*" DSC
#define CPTR10 " -5*" DSC
#define CPTR11 " -4*" DSC
#define CPTR12 " -3*" DSC
#define CPTR13 " -2*" DSC
#define CPTR14 " -1*" DSC
#define CPTR15 " "
#define CPTR16 " " DSC
#define CPTR17 " 2*" DSC
#define CPTR18 " 3*" DSC
#define CPTR19 " 4*" DSC
#define CPTR20 " 5*" DSC
#define CPTR21 " 6*" DSC
#define CPTR22 " 7*" DSC
#define CPTR23 " 8*" DSC
#define CPTR24 " 9*" DSC
#define CPTR25 " 10*" DSC
#define CPTR26 " 11*" DSC
#define CPTR27 " 12*" DSC
#define CPTR28 " 13*" DSC
#define CPTR29 " 14*" DSC

#endif

#define COLSIZE PNB "*" DS

#if defined(DREAL) || defined(DCPLX)
void ATL_USERMM(const int M, const int N, const int K, const double alpha, const double *A, const int lda, const double *B, const int ldb, const double beta, double *C, const int ldc)
#endif

#if defined(SREAL) || defined(SCPLX)
void ATL_USERMM(const int M, const int N, const int K, const float alpha, const float *A, const int lda, const float *B, const int ldb, const float beta, float *C, const int ldc)
#endif

{
  ASM ("push   %%ebp"::);
  ASM ("mov    %%esp,%%ebp"::);
  ASM ("push   %%ebx"::);
  ASM ("push   %%esi"::);
  ASM ("push   %%edi"::);
  #ifndef NOFEMMS
  ASM ("femms  "::);
  #endif

  /* Don't quite know why this computation is usefull, but it is
     used for the prefetching. It does not give the correct result for
     N cleanup.
   */
  ASM ("mov    $0x0,%%eax"::);
  ASM ("push   %%eax"::);
  ASM ("mov  " Astack ",%%eax"::);
  ASM ("add    $" DS "*" PNB "*" PNB ",%%eax"::);
  ASM ("mov  " Bstack ",%%ebx"::);
  ASM ("sub    %%ebx,%%eax"::);
  ASM ("push   %%eax"::);
  /* Load ldc onto stack and scale it.
     The scale factor must be 8 or lower because of limitiations in
     the intel adressing modes, so therefore it is multiplied by two
     for the complex case, as just scaling with 16 for DCPLX wouldn't
     work. Funny, eh! */
  ASM ("mov  " LDCstack ",%%eax"::);
  ASM ("lea    0x0(,%%eax," DS "),%%eax"::);
#if defined(DCPLX) || defined(SCPLX)
  ASM ("shl    $1,%%eax"::);
#endif
  ASM ("push   %%eax"::);

  ASM ("mov  " Nstack ", %%eax"::);
  ASM ("push   %%eax"::);
  /* Load &A, &B and &C into registers. */
  ASM ("mov  " Astack ",%%eax"::);
  ASM ("mov  " Bstack ",%%ebx"::);
  ASM ("mov  " Cstack ",%%ecx"::);
  /* Add offsets for small adresses. */
#ifdef DCPLX
  ASM ("add    $15*" DS ",%%ecx"::);
#else
  ASM ("add    $15*" DSC ",%%ecx"::);
#endif
  ASM ("add    $15*" DS ",%%ebx"::);
#ifdef BETA0
  ASM ("add    $5*" COLSIZE "+15*" DS ",%%eax"::);
  ASM ("mov    $6*" COLSIZE ",%%edx"::);
#elif defined(BETA1)
  ASM ("add    $5*" COLSIZE ",%%eax"::);
  ASM ("mov    $6*" COLSIZE "-15*" DS ",%%edx"::);
#else
  ASM ("add    $5*" COLSIZE "+15*" DS ",%%eax"::);
  ASM ("mov    $6*" COLSIZE ",%%edx"::);
  ASM (LOAD " " BETAstack::);  /* Initialise locbeta with beta */
  ASM (STO   " %0":"=m" (((locbeta))):);
#endif
  ASM ("push   %%eax"::);
  ASM ("push   %%ebp"::);
  ASM ("mov    $-1*" COLSIZE ",%%edi"::);
  ASM ("mov    $-3*" COLSIZE ",%%esi"::);
  ASM ("mov    $-5*" COLSIZE ",%%ebp"::);

  ASM (".align 16"::);
  ASM ("loopj_:"::);
#ifdef BETA0
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("nop    "::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("nop    "::);
  ASM ("repz"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
#elif defined(BETA1)
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM (ADD CPTR1 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR2 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (ADD CPTR0 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(1),%%st"::);
  ASM (ADD CPTR4 "(%%ecx)"::);
  ASM (LOAD " (%%eax)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR5 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(3)"::);
  ASM (LOAD CPTR3 "(%%ecx)"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM ("add    $15*" DS ",%%eax"::);
  ASM ("mov    %%edx,%%edx"::);
#else
  ASM (LOAD CPTR0 "(%%ecx)"::);
  ASM (LOAD CPTR1 "(%%ecx)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " %0"::"m" (((locbeta))));
  ASM ("fmul   %%st,%%st(2)"::);
  ASM ("fmul   %%st,%%st(1)"::);
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD CPTR4 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(2),%%st"::);
  ASM (LOAD CPTR5 "(%%ecx)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (LOAD CPTR3 "(%%ecx)"::);
  ASM ("fmul   %%st(4),%%st"::);
  ASM (LOAD CPTR2 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM ("fxch %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -15*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
#endif
  ASM (LOAD " -14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -13*" DS "(%%ebx)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -12*" DS "(%%ebx)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -11*" DS "(%%ebx)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -10*" DS "(%%ebx)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -9*" DS "(%%ebx)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -8*" DS "(%%ebx)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -7*" DS "(%%ebx)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -6*" DS "(%%ebx)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -5*" DS "(%%ebx)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -4*" DS "(%%ebx)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -3*" DS "(%%ebx)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -2*" DS "(%%ebx)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -1*" DS "(%%ebx)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%ebx)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " (%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 1*" DS "(%%ebx)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 2*" DS "(%%ebx)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 3*" DS "(%%ebx)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 4*" DS "(%%ebx)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 5*" DS "(%%ebx)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 6*" DS "(%%ebx)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 7*" DS "(%%ebx)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 8*" DS "(%%ebx)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 9*" DS "(%%ebx)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 10*" DS "(%%ebx)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 11*" DS "(%%ebx)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 12*" DS "(%%ebx)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 13*" DS "(%%ebx)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 14*" DS "(%%ebx)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " 14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("fxch   %%st(5)"::);
  ASM (STO CPTR0 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (STO CPTR1 "(%%ecx)"::);
  ASM ("fxch   %%st(1)"::);
  ASM (STO CPTR2 "(%%ecx)"::);
  ASM (STO CPTR3 "(%%ecx)"::);
  ASM (STO CPTR4 "(%%ecx)"::);
  ASM (STO CPTR5 "(%%ecx)"::);
  ASM ("add    %%edx,%%eax"::);
#ifdef BETA0
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("nop    "::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("nop    "::);
  ASM ("repz"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
#elif defined(BETA1)
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM (ADD CPTR7 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR8 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (ADD CPTR6 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(1),%%st"::);
  ASM (ADD CPTR10 "(%%ecx)"::);
  ASM (LOAD " (%%eax)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR11 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(3)"::);
  ASM (LOAD CPTR9 "(%%ecx)"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM ("add    $15*" DS ",%%eax"::);
  ASM ("mov    %%edx,%%edx"::);
#else
  ASM (LOAD CPTR6 "(%%ecx)"::);
  ASM (LOAD CPTR7 "(%%ecx)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " %0"::"m" (((locbeta))));
  ASM ("fmul   %%st,%%st(2)"::);
  ASM ("fmul   %%st,%%st(1)"::);
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD CPTR10 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(2),%%st"::);
  ASM (LOAD CPTR11 "(%%ecx)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (LOAD CPTR9 "(%%ecx)"::);
  ASM ("fmul   %%st(4),%%st"::);
  ASM (LOAD CPTR8 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM ("fxch %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -15*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
#endif
  ASM (LOAD " -14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -13*" DS "(%%ebx)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -12*" DS "(%%ebx)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -11*" DS "(%%ebx)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -10*" DS "(%%ebx)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -9*" DS "(%%ebx)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -8*" DS "(%%ebx)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -7*" DS "(%%ebx)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -6*" DS "(%%ebx)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -5*" DS "(%%ebx)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -4*" DS "(%%ebx)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -3*" DS "(%%ebx)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -2*" DS "(%%ebx)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -1*" DS "(%%ebx)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%ebx)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " (%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 1*" DS "(%%ebx)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 2*" DS "(%%ebx)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 3*" DS "(%%ebx)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 4*" DS "(%%ebx)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 5*" DS "(%%ebx)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 6*" DS "(%%ebx)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 7*" DS "(%%ebx)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 8*" DS "(%%ebx)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 9*" DS "(%%ebx)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 10*" DS "(%%ebx)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 11*" DS "(%%ebx)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 12*" DS "(%%ebx)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 13*" DS "(%%ebx)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 14*" DS "(%%ebx)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " 14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("fxch   %%st(5)"::);
#if defined(BETA1) && defined(USE_PREFETCH)
  ASM ("mov    %%ecx,0x14(%%esp,1)"::);
  ASM ("add    0xc(%%esp,1),%%ecx"::);
  ASM (Mstr(PREFETCHW) " (-2*64)(%%ecx)"::);
  ASM (Mstr(PREFETCHW) " (-64)(%%ecx)"::);
  ASM (Mstr(PREFETCHW) " (%%ecx)"::);
  ASM (Mstr(PREFETCHW) " (64)(%%ecx)"::);
  ASM ("nop    "::);
  ASM (Mstr(PREFETCHW) " (2*64-8)(%%ecx)"::);
  ASM ("mov    0x14(%%esp,1),%%ecx"::);
#endif
  ASM (STO CPTR6 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (STO CPTR7 "(%%ecx)"::);
  ASM ("fxch   %%st(1)"::);
  ASM (STO CPTR8 "(%%ecx)"::);
  ASM (STO CPTR9 "(%%ecx)"::);
  ASM (STO CPTR10 "(%%ecx)"::);
  ASM (STO CPTR11 "(%%ecx)"::);
  ASM ("add    %%edx,%%eax"::);
#ifdef DCPLX
    ASM ("add    $24*" DS ",%%ecx"::);
    ASM (".align 8"::);
#endif
#ifdef BETA0
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("nop    "::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("nop    "::);
  ASM ("repz"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
#elif defined(BETA1)
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM (ADD CPTR13 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR14 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (ADD CPTR12 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(1),%%st"::);
  ASM (ADD CPTR16 "(%%ecx)"::);
  ASM (LOAD " (%%eax)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR17 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(3)"::);
#ifndef DCPLX
  ASM ("repz"::);
#endif
  ASM (LOAD CPTR15 "(%%ecx)"::); /* CPTR15 is usually 0 */
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM ("add    $15*" DS ",%%eax"::);
  ASM ("mov    %%edx,%%edx"::);
#else
  ASM (LOAD CPTR12 "(%%ecx)"::);
  ASM (LOAD CPTR13 "(%%ecx)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " %0"::"m" (((locbeta))));
  ASM ("fmul   %%st,%%st(2)"::);
  ASM ("fmul   %%st,%%st(1)"::);
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD CPTR16 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(2),%%st"::);
  ASM (LOAD CPTR17 "(%%ecx)"::);
  ASM ("fmul   %%st(3),%%st"::);
#ifndef DCPLX
  ASM ("repz"::);
#endif
  ASM (LOAD CPTR15 "(%%ecx)"::);  /* CPTR15 is usually 0 */
  ASM ("fmul   %%st(4),%%st"::);
  ASM (LOAD CPTR14 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM ("fxch %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -15*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
#endif
  ASM (LOAD " -14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -13*" DS "(%%ebx)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -12*" DS "(%%ebx)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -11*" DS "(%%ebx)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -10*" DS "(%%ebx)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -9*" DS "(%%ebx)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -8*" DS "(%%ebx)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -7*" DS "(%%ebx)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -6*" DS "(%%ebx)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -5*" DS "(%%ebx)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -4*" DS "(%%ebx)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -3*" DS "(%%ebx)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -2*" DS "(%%ebx)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -1*" DS "(%%ebx)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%ebx)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " (%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 1*" DS "(%%ebx)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 2*" DS "(%%ebx)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 3*" DS "(%%ebx)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 4*" DS "(%%ebx)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 5*" DS "(%%ebx)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 6*" DS "(%%ebx)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 7*" DS "(%%ebx)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 8*" DS "(%%ebx)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 9*" DS "(%%ebx)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 10*" DS "(%%ebx)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 11*" DS "(%%ebx)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 12*" DS "(%%ebx)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 13*" DS "(%%ebx)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 14*" DS "(%%ebx)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " 14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("fxch   %%st(5)"::);
#ifdef USE_PREFETCH
  ASM (Mstr(PREFETCH) " (" COLSIZE "-2*64)(%%ebx)"::);
  ASM ("fnop   "::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (Mstr(PREFETCH) " (" COLSIZE "-1*64)(%%ebx)"::);
  ASM ("nop    "::);
  ASM (Mstr(PREFETCH) " (" COLSIZE "+0*64)(%%ebx)"::);
  ASM ("nop    "::);
  ASM (Mstr(PREFETCH) " (" COLSIZE "+1*64)(%%ebx)"::);
  ASM ("nop    "::);
  ASM (Mstr(PREFETCH) " (" COLSIZE "+2*64)(%%ebx)"::);
  ASM ("nop    "::);
#endif
  ASM (STO CPTR12 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (STO CPTR13 "(%%ecx)"::);
#ifndef DCPLX
  ASM ("repz"::); /* Counter later use of CPTR15 */
#endif
  ASM ("fxch %%st(1)"::);
  ASM (STO CPTR14 "(%%ecx)"::);
  ASM (STO CPTR15  "(%%ecx)"::);  /* CPTR15 is usually 0 */
  ASM (STO CPTR16 "(%%ecx)"::);
  ASM (STO CPTR17 "(%%ecx)"::);
  ASM ("add    %%edx,%%eax"::);
#ifdef BETA0
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("nop    "::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("nop    "::);
  ASM ("repz"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
#elif defined(BETA1)
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM (ADD CPTR19 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR20 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (ADD CPTR18 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(1),%%st"::);
  ASM (ADD CPTR22 "(%%ecx)"::);
  ASM (LOAD " (%%eax)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR23 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(3)"::);
  ASM (LOAD CPTR21 "(%%ecx)"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM ("add    $15*" DS ",%%eax"::);
  ASM ("mov    %%edx,%%edx"::);
#else
  ASM (LOAD CPTR18 "(%%ecx)"::);
  ASM (LOAD CPTR19 "(%%ecx)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " %0"::"m" (((locbeta))));
  ASM ("fmul   %%st,%%st(2)"::);
  ASM ("fmul   %%st,%%st(1)"::);
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD CPTR22 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(2),%%st"::);
  ASM (LOAD CPTR23 "(%%ecx)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (LOAD CPTR21 "(%%ecx)"::);
  ASM ("fmul   %%st(4),%%st"::);
  ASM (LOAD CPTR20 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM ("fxch %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -15*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
#endif
  ASM (LOAD " -14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -13*" DS "(%%ebx)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -12*" DS "(%%ebx)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -11*" DS "(%%ebx)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -10*" DS "(%%ebx)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -9*" DS "(%%ebx)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -8*" DS "(%%ebx)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -7*" DS "(%%ebx)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -6*" DS "(%%ebx)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -5*" DS "(%%ebx)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -4*" DS "(%%ebx)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -3*" DS "(%%ebx)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -2*" DS "(%%ebx)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -1*" DS "(%%ebx)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%ebx)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " (%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 1*" DS "(%%ebx)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 2*" DS "(%%ebx)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 3*" DS "(%%ebx)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 4*" DS "(%%ebx)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 5*" DS "(%%ebx)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 6*" DS "(%%ebx)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 7*" DS "(%%ebx)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 8*" DS "(%%ebx)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 9*" DS "(%%ebx)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 10*" DS "(%%ebx)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 11*" DS "(%%ebx)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 12*" DS "(%%ebx)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 13*" DS "(%%ebx)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 14*" DS "(%%ebx)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " 14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("fxch   %%st(5)"::);
  ASM (STO CPTR18 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (STO CPTR19 "(%%ecx)"::);
  ASM ("fxch   %%st(1)"::);
  ASM (STO CPTR20 "(%%ecx)"::);
  ASM (STO CPTR21 "(%%ecx)"::);
  ASM (STO CPTR22 "(%%ecx)"::);
  ASM (STO CPTR23 "(%%ecx)"::);
  ASM ("add    %%edx,%%eax"::);
#ifdef DCPLX
  ASM ("add    $24*" DS ",%%ecx"::);
  ASM (".align 8"::);
#endif
#ifdef BETA0
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("nop    "::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("nop    "::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("nop    "::);
  ASM ("repz"::);
  ASM (MUL  " -15*" DS "(%%ebx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
#elif defined(BETA1)
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM (ADD CPTR25 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR26 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (ADD CPTR24 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(1),%%st"::);
  ASM (ADD CPTR28 "(%%ecx)"::);
  ASM (LOAD " (%%eax)"::);
  ASM ("fmul   %%st(2),%%st"::);
  ASM (ADD CPTR29 "(%%ecx)"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(3)"::);
  ASM (LOAD CPTR27 "(%%ecx)"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
  ASM ("add    $15*" DS ",%%eax"::);
  ASM ("mov    %%edx,%%edx"::);
#else
  ASM (LOAD CPTR24 "(%%ecx)"::);
  ASM (LOAD CPTR25 "(%%ecx)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " %0"::"m" (((locbeta))));
  ASM ("fmul   %%st,%%st(2)"::);
  ASM ("fmul   %%st,%%st(1)"::);
  ASM (LOAD " -15*" DS "(%%ebx)"::);
  ASM (LOAD CPTR28 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmul %%st(2),%%st"::);
  ASM (LOAD CPTR29 "(%%ecx)"::);
  ASM ("fmul   %%st(3),%%st"::);
  ASM (LOAD CPTR27 "(%%ecx)"::);
  ASM ("fmul   %%st(4),%%st"::);
  ASM (LOAD CPTR26 "(%%ecx)"::);
  ASM ("repz"::);
  ASM ("fmulp %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM ("fxch %%st(3)"::);
  ASM ("mov    %%edx,%%edx"::);
  ASM (LOAD " -15*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -15*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -15*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -14*" DS "(%%ebx)"::);
#endif
  ASM (LOAD " -14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -13*" DS "(%%ebx)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -12*" DS "(%%ebx)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -11*" DS "(%%ebx)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -10*" DS "(%%ebx)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -9*" DS "(%%ebx)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -8*" DS "(%%ebx)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -7*" DS "(%%ebx)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -6*" DS "(%%ebx)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -5*" DS "(%%ebx)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -4*" DS "(%%ebx)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -3*" DS "(%%ebx)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -2*" DS "(%%ebx)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " -1*" DS "(%%ebx)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " -1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " -1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%ebx)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM ("repz"::);
  ASM (LOAD " (%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " (%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 1*" DS "(%%ebx)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 1*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 1*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 2*" DS "(%%ebx)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 2*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 2*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 3*" DS "(%%ebx)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 3*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 3*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 4*" DS "(%%ebx)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 4*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 4*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 5*" DS "(%%ebx)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 5*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 5*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 6*" DS "(%%ebx)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 6*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 6*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 7*" DS "(%%ebx)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 7*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 7*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 8*" DS "(%%ebx)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 8*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 8*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 9*" DS "(%%ebx)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 9*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 9*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 10*" DS "(%%ebx)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 10*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 10*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 11*" DS "(%%ebx)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 11*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 11*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 12*" DS "(%%ebx)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 12*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 12*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 13*" DS "(%%ebx)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 13*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM (MUL  " 13*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM (LOAD " 14*" DS "(%%ebx)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%ebp,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(7)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,4)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(6)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%esi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(5)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,2)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(4)"::);
  ASM (LOAD " 14*" DS "(%%eax,%%edi,1)"::);
  ASM ("fmul   %%st(1),%%st"::);
  ASM ("faddp  %%st,%%st(3)"::);
  ASM ("repz"::);
  ASM (MUL  " 14*" DS "(%%eax)"::);
  ASM ("faddp  %%st,%%st(1)"::);
  ASM ("fxch   %%st(5)"::);
#ifdef USE_PREFETCH
  ASM ("mov    %%edx,0x14(%%esp,1)"::);
  ASM ("mov    0x10(%%esp,1),%%edx"::);
  ASM ("lea    (%%edx,%%ebx,1),%%edx"::);
  ASM (Mstr(PREFETCH) " (-2*64)(%%edx)"::);
  ASM ("nop    "::);
  ASM (Mstr(PREFETCH) " (-1*64)(%%edx)"::);
  ASM (Mstr(PREFETCH) " (+0*64)(%%edx)"::);
  ASM ("nop    "::);
  ASM (Mstr(PREFETCH) " (+1*64)(%%edx)"::);
  ASM (Mstr(PREFETCH) " (+2*64-8)(%%edx)"::);
  ASM ("mov    0x14(%%esp,1),%%edx"::);
  ASM ("mov    %%eax,%%eax"::);
  ASM ("fnop   "::);
#endif
  ASM (STO CPTR24 "(%%ecx)"::);
  ASM ("fxch   %%st(3)"::);
  ASM (STO CPTR25 "(%%ecx)"::);
  ASM ("fxch   %%st(1)"::);
  ASM (STO CPTR26 "(%%ecx)"::);
  ASM (STO CPTR27 "(%%ecx)"::);
  ASM (STO CPTR28 "(%%ecx)"::);
  ASM (STO CPTR29 "(%%ecx)"::);
#ifdef DCPLX
  ASM ("sub    $48*" DS ",%%ecx"::);
#endif
  ASM ("sub    %%edi,%%ebx"::);        /* Advance B one column. */
  ASM ("mov    0x4(%%esp,1),%%eax"::); /* Reset A */
  ASM ("add    0xc(%%esp,1),%%ecx"::); /* Advance C */
  ASM ("decl   0x8(%%esp,1)"::);       /* Decrease counter. */
  ASM ("jne    loopj_"::);

  ASM ("end_:"::);
  #ifndef NOFEMMS
  ASM ("femms  "::);
  #endif
  ASM ("pop    %%ebp"::);
  ASM ("add    $0x14,%%esp"::);
  ASM ("pop    %%edi"::);
  ASM ("pop    %%esi"::);
  ASM ("pop    %%ebx"::);
  ASM ("leave  "::);
}
